{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "# Model Management Demo"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "Before managing models, we need to prepare a model by training. In this example, we use the homo_logistic_regression(https://github.com/FederatedAI/FATE/tree/master/examples/federatedml-1.x-examples/homo_logistic_regression example. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'data': {'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011073298433874&role=local&party_id=0', 'job_dsl_path': '/data/projects/fate/python/jobs/2020042011073298433874/job_dsl.json', 'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011073298433874/job_runtime_conf.json', 'logs_directory': '/data/projects/fate/python/logs/2020042011073298433874', 'namespace': 'homo_breast_guest', 'table_name': 'homo_breast_guest'}, 'jobId': '2020042011073298433874', 'retcode': 0, 'retmsg': 'success'}\n",
            "running,running,running\n",
            "running,running,running\n",
            "running,running,running\n",
            "running,success,success\n",
            "running,success,success\n",
            "running,success,success\n",
            "success,success,success\n",
            "Success\n",
            "Success!\n",
            "{\n",
            "    'data': {\n",
            "        'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011074074717177&role=guest&party_id=10000',\n",
            "        'job_dsl_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_dsl.json',\n",
            "        'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_runtime_conf.json',\n",
            "        'logs_directory': '/data/projects/fate/python/logs/2020042011074074717177',\n",
            "        'model_info': {\n",
            "            'model_id': 'arbiter-10000#guest-10000#host-10000#model',\n",
            "            'model_version': '2020042011074074717177'\n",
            "        }\n",
            "    },\n",
            "    'jobId': '2020042011074074717177',\n",
            "    'retcode': 0,\n",
            "    'retmsg': 'success'\n",
            "}\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "running\n",
            "success\n",
            "Success\n",
            "Success!\n",
            "{\n",
            "    'data': {\n",
            "        'board_url': 'http://fateboard:8080/index.html#/dashboard?job_id=2020042011074074717177&role=guest&party_id=10000',\n",
            "        'job_dsl_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_dsl.json',\n",
            "        'job_runtime_conf_path': '/data/projects/fate/python/jobs/2020042011074074717177/job_runtime_conf.json',\n",
            "        'logs_directory': '/data/projects/fate/python/logs/2020042011074074717177',\n",
            "        'model_info': {\n",
            "            'model_id': 'arbiter-10000#guest-10000#host-10000#model',\n",
            "            'model_version': '2020042011074074717177'\n",
            "        }\n",
            "    },\n",
            "    'jobId': '2020042011074074717177',\n",
            "    'retcode': 0,\n",
            "    'retmsg': 'success'\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "import fml_manager\n",
        "import json, time, requests\n",
        "import os\n",
        "from fml_manager import *\n",
        "\n",
        "manager = fml_manager.FMLManager()\n",
        "response = manager.load_data('examples/data/breast_homo_guest.csv', 'homo_breast_guest', 'homo_breast_guest', 1, 1, 10, api_version='1.4')\n",
        "output = json.loads(response.content)\n",
        "print(output)\n",
        "guest_job_id = output['jobId']\n",
        "guest_query_condition = {\n",
        "    'job_id':guest_job_id\n",
        "}\n",
        "response = manager.load_data('examples/data/breast_homo_host.csv', 'homo_breast_host', 'homo_breast_host', 1, 1, 10, api_version='1.4')\n",
        "output = json.loads(response.content)\n",
        "host_job_id = output['jobId']\n",
        "host_query_condition = {\n",
        "    'job_id':host_job_id\n",
        "}\n",
        "response = manager.load_data('examples/data/breast_homo_test.csv', 'homo_breast_test', 'homo_breast_test', 1, 1, 10, api_version='1.4')\n",
        "output = json.loads(response.content)\n",
        "test_job_id = output['jobId']\n",
        "test_query_condition = {\n",
        "    'job_id':test_job_id\n",
        "}\n",
        "\n",
        "for i in range(500):\n",
        "    time.sleep(1)\n",
        "    guest_status = manager.query_job(guest_query_condition).json()['data'][0]['f_status']\n",
        "    host_status = manager.query_job(host_query_condition).json()['data'][0]['f_status']\n",
        "    test_status = manager.query_job(host_query_condition).json()['data'][0]['f_status']\n",
        "    \n",
        "    print('{},{},{}'.format(guest_status, host_status, test_status))\n",
        "    \n",
        "    if guest_status == 'failed' or host_status == 'failed' or test_status == 'failed':\n",
        "        print('Failed')\n",
        "        raise Exception('Failed to run the jobs')\n",
        "    if guest_status == 'success' and host_status == 'success' and test_status == 'success':\n",
        "        print('Success')       \n",
        "        break\n",
        "\n",
        "\n",
        "# dsl\n",
        "data_io = ComponentBuilder()\\\n",
        "    .with_name('dataio_0')\\\n",
        "    .with_module('DataIO')\\\n",
        "    .add_input_data('args.train_data')\\\n",
        "    .add_output_data('train')\\\n",
        "    .add_output_model('dataio').build()\n",
        "        \n",
        "\n",
        "homo_lr = ComponentBuilder()\\\n",
        "    .with_name('homo_lr_0')\\\n",
        "    .with_module('HomoLR')\\\n",
        "    .add_input_train_data('dataio_0.train')\\\n",
        "    .add_output_data('train')\\\n",
        "    .add_output_model('homolr').build()\n",
        "\n",
        "evaluation = ComponentBuilder()\\\n",
        "    .with_name('evaluation_0')\\\n",
        "    .with_module('Evaluation')\\\n",
        "    .add_input_data('homo_lr_0.train')\\\n",
        "    .ad_output_data('evaluate')\\\n",
        "    .with_need_deploy(False).build()\n",
        "\n",
        "pipeline = Pipline()\n",
        "    data_io, \n",
        "    hetero_lr, \n",
        "    evaluation\n",
        ")\n",
        "\n",
        "# Configuration\n",
        "initiator = Initiator(role='guest', party_id=10000)\n",
        "\n",
        "job_parameters = JobParameters(work_mode=1)\n",
        "\n",
        "role = RoleBuilder()\\\n",
        "    .add_guest(10000)\\\n",
        "    .add_host(10000)\\\n",
        "    .add_arbiter(10000).build()\n",
        "\n",
        "eval_config = {\n",
        "       'need_run': [False]\n",
        " }\n",
        "\n",
        "role_parameters = RoleParametersBuilder()\\\n",
        "    .add_guest_train_data(namespace='homo_breast_guest', name='homo_breast_guest')\\\n",
        "    .add_host_train_data(namespace='homo_breast_host', name='homo_breast_host')\\\n",
        "    .add_host_module_config(module='evalution_0', config=eval_config).build()\n",
        "\n",
        "homo_lr_params = {\n",
        "            'penalty': 'L2',\n",
        "            'optimizer': 'sgd',\n",
        "            'eps': 1e-5,\n",
        "            'alpha': 0.01,\n",
        "            'max_iter': 10,\n",
        "            'converge_func': 'diff',\n",
        "            'batch_size': 500,\n",
        "            'learning_rate': 0.15,\n",
        "            'decay': 1,\n",
        "            'decay_sqrt': True,\n",
        "            'init_param': {\n",
        "                'init_method': 'zeros'\n",
        "            },\n",
        "            'encrypt_param': {\n",
        "                'method': 'Paillier'\n",
        "            },\n",
        "            'cv_param': {\n",
        "                'n_splits': 4,\n",
        "                'shuffle': True,\n",
        "                'random_seed': 33,\n",
        "                'need_cv': False\n",
        "            }\n",
        "        }\n",
        "\n",
        "dotaio_config = {\n",
        "            'with_label': True,\n",
        "            'label_name': 'y',\n",
        "            'label_type': 'int',\n",
        "            'output_format': 'dense'\n",
        "        },\n",
        "\n",
        "algorithm_parameters = AlgorithmParametersBuilder()\\\n",
        "    .add_module_config(module='homo_lr_0', config=homo_lr_params)\n",
        "    .add_model_config(model='dataio_0', config=dotaio_config).build()\n",
        "\n",
        "config = ConfigBuilder()\\\n",
        "    .with_initiator(initiator)\\\n",
        "    .with_job_parameters(job_parameters)\\\n",
        "    .with_role(role)\\\n",
        "    .with_role_parameters(role_parameters)\\\n",
        "    .with_algorithm_parameters(algorithm_parameters).build()\n",
        "\n",
        "\n",
        "response = manager.submit_job(pipeline.to_dict(),config.to_dict())\n",
        "manager.prettify(response, True)\n",
        "stdout = json.loads(response.content)\n",
        "jobId = stdout['jobId']\n",
        "query_condition = {\n",
        "    'job_id':jobId\n",
        "}\n",
        "\n",
        "model_id, model_version = '', ''\n",
        "for i in range(500):\n",
        "    time.sleep(1)\n",
        "    job_detail = manager.query_job(query_condition).json()\n",
        "    final_status = job_detail['data'][0]['f_status']\n",
        "    print(final_status)\n",
        "    \n",
        "    if final_status == 'failed':\n",
        "        print('Failed')\n",
        "        manager.prettify(job_detail, True)\n",
        "        response = manager.fetch_job_log(jobId)\n",
        "        raise Exception('Failed to run the job')\n",
        "    if final_status == 'success':\n",
        "        print('Success')\n",
        "        manager.prettify(response, True)\n",
        "        output = json.loads(response.content)\n",
        "        model_id, model_version = output['data']['model_info']['model_id'], output['data']['model_info']['model_version']\n",
        "        break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "Print existed model, the API is: ```print_model_version(self, role, party_id, model_id)```"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Success!\n",
            "{\n",
            "    'data': [\n",
            "        {\n",
            "            'commitId': '2020042011074074717177',\n",
            "            'log': '[AUTO] save model at 2020-04-20 11:09:40.165231.',\n",
            "            'name': '2020042011074074717177',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042011022342527372',\n",
            "            'repeatCommit': true,\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042011022342527372',\n",
            "            'log': '[AUTO] save model at 2020-04-20 11:06:17.494397.',\n",
            "            'name': '2020042011022342527372',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042010084278627567',\n",
            "            'repeatCommit': true,\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042010084278627567',\n",
            "            'log': '[AUTO] save model at 2020-04-20 10:13:33.310625.',\n",
            "            'name': '2020042010084278627567',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042009122786739056',\n",
            "            'repeatCommit': true,\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042009122786739056',\n",
            "            'log': '[AUTO] save model at 2020-04-20 10:04:07.545335.',\n",
            "            'name': '2020042009122786739056',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042009081081759152',\n",
            "            'repeatCommit': true,\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042009081081759152',\n",
            "            'log': '[AUTO] save model at 2020-04-20 09:10:29.357328.',\n",
            "            'name': '2020042009081081759152',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008513api_version='1.4'681344',\n",
            "            'repeatCommit': true,\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042008513api_version='1.4'681344',\n",
            "            'log': '[AUTO] save model at 2020-04-20 08:5api_version='1.4'5.427548.',\n",
            "            'name': '2020042008513api_version='1.4'681344',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008495252468240',\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042008495252468240',\n",
            "            'log': '[AUTO] save model at 2020-04-20 08:49:56.606507.',\n",
            "            'name': '2020042008495252468240',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008494205128236',\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042008494205128236',\n",
            "            'log': '[AUTO] save model at 2020-04-20 08:49:46.874495.',\n",
            "            'name': '2020042008494205128236',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008480590014232',\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042008480590014232',\n",
            "            'log': '[AUTO] save model at 2020-04-20 08:48:10.868260.',\n",
            "            'name': '2020042008480590014232',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008464736494228',\n",
            "            'tag': null\n",
            "        },\n",
            "        {\n",
            "            'commitId': '2020042008464736494228',\n",
            "            'log': '[AUTO] save model at 2020-04-20 08:46:52.258443.',\n",
            "            'name': '2020042008464736494228',\n",
            "            'namespace': 'guest#10000#arbiter-10000#guest-10000#host-10000#model',\n",
            "            'parent': '2020042008460920911924',\n",
            "            'tag': null\n",
            "        }\n",
            "    ],\n",
            "    'retcode': 0,\n",
            "    'retmsg': 'success'\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "if model_id != '':\n",
        "    response = manager.print_model_version('guest','10000', model_id)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "Output the model, the API is ```model_output(self, role, party_id, model_id, model_version，model_component)```. The ```model_component``` is what you defined in training conf. The output is base64 encoding, which need decoding and parsing back."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'metadata': 'CgJMMhHxaOOItfjkPhl7FK5H4XqEPyIDc2dkMPQDOTMzMzMzM8M/QApKBGRpZmZQAlgB', 'parameters': 'CAoSUP4QBQQ+79o/Y+ScSvKa1j+RTreM/XfUPyk/T2KsNtM/ik0jT2po0j+lMOUXo9zRPwWjqz1MetE/3VWUfhkz0T8KB2etSv7QP52mdO1A1tA/Ig0KAngwEbFDSoWA/sG/Ig0KAngxEcob/dudLr6/Ig0KAngyEWYZ+hcRpMG/Ig0KAngzEfmv/7Rc+r+/Ig0KAng0EbDD/W9RW7m/Ig0KAng1EbyOMlppJri/Ig0KAng2EbMEbULVqru/Ig0KAng3ERQQVXbn+MG/Ig0KAng4EW6/Nt4yu7W/Ig0KAng5EQZZbnGtrKW/Ig4KA3gxMBEsGFvBSI3AvyIOCgN4MTERZZM+KKAGur8iDgoDeDEyEc6PglYzgMC/Ig4KA3gxMxHa0QRjolK+vyIOCgN4MTQR9tqVWW+Fr78iDgoDeDE1EUAimmbYbLO/Ig4KA3gyMBG6wwtUoXG0vyIOCgN4MTYRH2mQvHuRur8iDgoDeDIxEdeP/oYJfnY/Ig4KA3gxNxFSVtQreM/AvyIOCgN4MjIRzAmY7no3sr8iDgoDeDIzEcwFGjerVbK/Ig4KA3gxOBHDXhlmJQ2rvyIOCgN4MjQRoPtqyzc7iD8iDgoDeDE5EXaSYAuFHaM/Ig4KA3gyNRGNsnK5wSqBvyIOCgN4MjYRSLDRsrf0Gj8iDgoDeDI3EUcfkMZaT6a/Ig4KA3gyOBHYUPahnWidPyIOCgN4MjkRXdrDc3/ooD8pCQy8iYha4T8yAngwMgJ4MTICeDIyAngzMgJ4NDICeDUyAng2MgJ4NzICeDgyAng5MgN4MTAyA3gxMTIDeDEyMgN4MTMyA3gxNDIDeDE1MgN4MTYyA3gxNzIDeDE4MgN4MTkyA3gyMDIDeDIxMgN4MjIyA3gyMzIDeDI0MgN4MjUyA3gyNjIDeDI3MgN4MjgyA3gyOQ=='}\n"
          ]
        }
      ],
      "source": [
        "response = manager.model_output('guest','10000', model_id, model_version, 'homo_lr_0.homolr:HomoLogisticRegression')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "And we can try offline prediction feature."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'data':{'board_url':'http://fateboard:8080/index.html#/dashboard?job_id=2020042011api_version='1.4'807243678&role=guest&party_id=10000','job_dsl_path':'/data/projects/fate/python/jobs/2020042011103807243678/job_dsl.json','job_runtime_conf_path':'/data/projects/fate/python/jobs/2020042011103807243678/job_runtime_conf.json','logs_directory':'/data/projects/fate/python/logs/2020042011103807243678','model_info':{'model_id':'arbiter-10000#guest-10000#host-10000#model','model_version':'2020042011074074717177'}},'jobId':'2020042011103807243678','retcode':0,'retmsg':'success'}\n",
            "\n"
          ]
        }
      ],
      "source": [
        "is_vertical = False\n",
        "initiator_party_role = 'guest'\n",
        "initiator_party_id = '10000'\n",
        "work_mode = 1\n",
        "federated_roles = {\n",
        "        'guest': [10000],\n",
        "        'host': [10000],\n",
        "        'arbiter': [10000]\n",
        "}\n",
        "guest_data_name = 'homo_breast_test'\n",
        "guest_data_namespace = 'homo_breast_test'\n",
        "host_data_name = 'homo_breast_test'\n",
        "host_data_namespace = 'homo_breast_test'\n",
        "\n",
        "response = manager.offline_predict_on_dataset(is_vertical, initiator_party_role, initiator_party_id, work_mode, model_id, model_version, federated_roles, guest_data_name, guest_data_namespace, host_data_name, host_data_namespace)\n",
        "print(response.text)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "pycharm": {}
      },
      "source": [
        "The result can be checked in FATE-Board."
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}